const puppeteer = require("puppeteer");
const path = require("path");
const fs = require("fs");

let scrape = async () => {
  const browser = await puppeteer.launch({
    headless: true,
    args: [
      "--no-sandbox",
      "--disable-setuid-sandbox",
      "--disable-blink-features=AutomationControlled",
    ],
    dumpio: false,
  });
  const page = await browser.newPage();
  await page.goto("https://www.toutiao.com/", {
    waitUntil: "load",
    timeout: 0,
  });
  await page.waitForSelector(".main-content"); // 非常重要，判断等一个节点加载完成后再往下执行
  // 可以制造向下滚屏效果，更多的获取数据
  await page.evaluate(() => {
    window.scrollBy(0, document.body.scrollHeight);
  });
  await page.waitForTimeout(1000);

  await page.evaluate(() => {
    window.scrollBy(0, document.body.scrollHeight);
  });
  await page.waitForTimeout(1000);

  await page.evaluate(() => {
    window.scrollBy(0, document.body.scrollHeight);
  });
  await page.waitForTimeout(1000);

  await page.evaluate(() => {
    window.scrollBy(0, document.body.scrollHeight);
  });
  await page.waitForTimeout(1000);

  const result = await page.evaluate(() => {
    window.scrollBy(0, document.body.scrollHeight);
    let content = document.querySelectorAll(
      ".main-content>.left-container>.show-monitor>div>.ttp-feed-module>div>.feed-card-wrapper"
    );
    // let price = document.querySelector(".price_color").innerText;
    return Array.from(content)
      .map((item) => {
        if (item.childNodes[0].childNodes[0].childNodes[0].innerText) {
          return {
            title: item.childNodes[0].childNodes[0].childNodes[0].innerText,
            href: item.childNodes[0].childNodes[0].childNodes[0].attributes[0]
              .value,
          };
        }
      })
      .filter((item) => !!item);
  });
  // Scrape

  browser.close();
  return result;
};

scrape()
  .then((value) => {
    // console.log(value); // 成功！
    fs.writeFileSync(
      path.join(__dirname, "./toutiao.json"),
      JSON.stringify(value),
      "utf8"
    );
  })
  .catch((err) => {
    console.log(err);
  });
